// ExtractorApi.cpp : Defines the entry point for the console application.
//

#include "stdafx.h"
#include "ExtractorApi.h"

#include <map>
#include <vector>
#include <string>
using namespace std;

#include <windows.h>

//extern "C" {
#include "extr_api.h"
//};

#include <mshtml.h>

#ifdef _DEBUG
#define new DEBUG_NEW
#undef THIS_FILE
static char THIS_FILE[] = __FILE__;
#endif

/////////////////////////////////////////////////////////////////////////////
// The one and only application object

CWinApp theApp;

using namespace std;

/*
bool operator < (  string First  ,  string Second )
{
	if ( strcmp(First.c_str(),Second.c_str()) < 0 )
		return true;
	else
		return false;
}
bool operator == (  string First  ,  string Second )
{
	if ( strcmp(First.c_str(),Second.c_str()) == 0 )
		return true;
	else
		return false;
}
bool operator != (  string First  ,  string Second )
{
	if ( strcmp(First.c_str(),Second.c_str()) != 0 )
		return true;
	else
		return false;
}
*/

class KeyPhrase{

public:

	long index;
	string Phrase;
	KeyPhrase *left,*right,*next;
	KeyPhrase(){};
	KeyPhrase(int lcIndex,string lcPhrase){index = lcIndex; Phrase = lcPhrase; right=NULL;left=NULL;next=NULL;}
};


//typedef map<string,KeyPhrase> KeyPhraseMap;


class FeatureItem{

public:

	long Index;
	double Weight;
	FeatureItem(){};
	FeatureItem(int LcIndex , double LcWeight){ Index=LcIndex ; Weight = LcWeight;}
};


typedef map<int,FeatureItem> FeatureItemMap;


class HtmlPage
{

public:
	FeatureItemMap MyFeatures;

	bool GenerateTermsList(char * stream);
	char* ReadFile(string FilePath);
	char* IncreaseArray(char *TempStream, int Chunk);
	void AddPage(string PageUrl);
};

typedef vector<HtmlPage> HtmlPageVector;


//KeyPhraseMap AllPhrasesMap;
HtmlPageVector AllPagesVector;
KeyPhrase PhrasesTree;
vector<KeyPhrase> PhrasesVector;
int NumOfPhrases;

int AddKeyPhrase( string strPhrase )
{
	KeyPhrase *ptrKeyPhrase = NULL;
	int Result = 0;
	ptrKeyPhrase = &PhrasesTree;	
	if ( NumOfPhrases == 1 )
	{
		ptrKeyPhrase->next = new KeyPhrase( NumOfPhrases ,strPhrase );
		PhrasesVector.push_back(*ptrKeyPhrase->next);
		return NumOfPhrases++;
	}
	ptrKeyPhrase=ptrKeyPhrase->next;
	while ( ptrKeyPhrase )
	{
		Result = strcmp(ptrKeyPhrase->Phrase.c_str() , strPhrase.c_str() );
		if ( Result == 0 )
			return ptrKeyPhrase->index;
		else if ( Result > 0 )
		{
			if ( ptrKeyPhrase->left != NULL )
				ptrKeyPhrase = ptrKeyPhrase->left;
			else
			{
				ptrKeyPhrase->left = new KeyPhrase( NumOfPhrases ,strPhrase );
				PhrasesVector.push_back(*ptrKeyPhrase->left);
				return NumOfPhrases++;
			}
		}
		else
		{
			if ( ptrKeyPhrase->right != NULL )
				ptrKeyPhrase = ptrKeyPhrase->right;
			else
			{
				ptrKeyPhrase->right = new KeyPhrase( NumOfPhrases , strPhrase );
				PhrasesVector.push_back(*ptrKeyPhrase->right);
				return NumOfPhrases++;
			}
		}		
	}
	return 0;
}

int SearchDirectory(vector<string> &refvecFiles,
                    const string   &refcstrRootDirectory,
                    const string   &refcstrExtension)
{
  string          strFilePath;             // Filepath
  string          strPattern;              // Pattern
  HANDLE          hFile;                   // Handle to file
  WIN32_FIND_DATA FileInformation;         // File information


  strPattern = refcstrRootDirectory + "\\*." + refcstrExtension;

  hFile = ::FindFirstFile(strPattern.c_str(), &FileInformation);
  if(hFile != INVALID_HANDLE_VALUE)
  {
    do
    {
      if(FileInformation.cFileName[0] != '.')
      {
        strFilePath.erase();
        strFilePath = refcstrRootDirectory + "\\" + FileInformation.cFileName;

        if(FileInformation.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY)
        {
          // Search subdirectory
          int iRC = SearchDirectory(refvecFiles, strFilePath, refcstrExtension);
          if(iRC)
            return iRC;
        }
        else
        {
          // Save filename
          refvecFiles.push_back(strFilePath);
        }
      }
    } while(::FindNextFile(hFile, &FileInformation) == TRUE);

    // Close handle
    ::FindClose(hFile);

    DWORD dwError = ::GetLastError();
    if(dwError != ERROR_NO_MORE_FILES)
      return dwError;
  }

  return 0;
}

bool HtmlPage::GenerateTermsList(char * stream)
{
	bool allWentWell = true ;
	
	int    LanguageID    = 0;

	int    OutCharCodeID = 0;
	double DesiredNumber = 30;

	void  *StopMemory;
	void  *DocumentMemory;

	int    PhraseListSize;
	int    PhraseIndex;
	char  *Phrase;
	double Score;

	int    ErrorCode;
	char  *ErrorMessage;
	
	char   num_phrase_string[IN_LEN];
	string strPhrase;
	int  Result=0;
	FeatureItem *pFeatureItem=NULL;
//	KeyPhraseMap::iterator posKeyPhrase;
//  if (!IsValidPage())
  //  return false ;

//  m_lstPageTMyTerms->Clear() ;

	ErrorCode = ExtrCreateStopMemory(&StopMemory);
	if (ErrorCode > 0) {
		ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
	       	//AddToLog (5, ErrorMessage);
		return false ;
	}

	ErrorCode = ExtrCreateDocumentMemory(&DocumentMemory);

	if (ErrorCode > 0) {
		ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
	        //AddToLog (5, ErrorMessage);
		return false ;
	}

	ErrorCode = ExtrActivateHTMLFilter(DocumentMemory, StopMemory);

    	if (ErrorCode > 0) {
		ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
	        //AddToLog (5, ErrorMessage);
		return false ;
    	}


	strcpy (num_phrase_string,"30") ; // maximum is 30

	num_phrase_string[strcspn(num_phrase_string, "\n\r")] = '\0';

	if (num_phrase_string[0] == '\0') {
	       //AddToLog (6, "\nThe desired number of phrases is set to seven (7). \n"); 
	} else {

		ErrorCode = ExtrSetNumberPhrases(DocumentMemory, DesiredNumber);
		if (ErrorCode > 0) {
			ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
		    	//AddToLog (5, ErrorMessage);
			return false ;
	    }
	}

    	OutCharCodeID = ENC_DOS;
    	ErrorCode = ExtrSetOutputCode(DocumentMemory, OutCharCodeID);
    	if (ErrorCode > 0) {
        	ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
          	//AddToLog (5, ErrorMessage);
          	return false ;
    	}

	string m_strHTMLBody;
   	ErrorCode = ExtrReadDocumentBuffer(DocumentMemory, StopMemory,
                        (void *) stream, strlen(stream));
   	if (ErrorCode > 0) {
   		ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
   		//AddToLog (5, ErrorMessage);
   		return false ;
   	}

	ErrorCode = ExtrSignalDocumentEnd(DocumentMemory, StopMemory);

	if (ErrorCode > 0) {
		ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
	      	//AddToLog (5, ErrorMessage);
		return false ;
	}

	ErrorCode = ExtrGetDocumentLanguage(DocumentMemory, &LanguageID);

	if (ErrorCode > 0) {
		ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
	       	//AddToLog (5, ErrorMessage);
		return false ;
	}

	ErrorCode = ExtrGetPhraseListSize(DocumentMemory, &PhraseListSize);

	if (ErrorCode > 0) {
		ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
		//AddToLog (5, ErrorMessage);
		return false ;
	}

	if (PhraseListSize == 0) {
	       //AddToLog (3, "\nNo keyphrases were extracted! \n\n");
	}

	if (PhraseListSize > 0) {
	    for (PhraseIndex = 0; PhraseIndex < PhraseListSize; PhraseIndex++) 
		{
			ErrorCode = ExtrGetPhraseByIndex(DocumentMemory, PhraseIndex,
									        (void **) &Phrase);

			
			strPhrase.assign(Phrase);
			
			 
			if (ErrorCode > 0) {
				ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
				//AddToLog (5, ErrorMessage);
				return false ;
			}
			ErrorCode = ExtrGetScoreByIndex(DocumentMemory, PhraseIndex, &Score);

			Result=AddKeyPhrase(strPhrase);
			pFeatureItem=new FeatureItem(Result , Score );
			MyFeatures.insert(FeatureItemMap::value_type(Result,*pFeatureItem));
			delete pFeatureItem;
			if (ErrorCode > 0) {
				ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
		       		//AddToLog (5, ErrorMessage);
				return false ;
			}
			cout<<PhraseIndex<<" "<<Phrase<<" "<<Score<<endl;
	//		m_lstPageTMyTerms->Add(new TMyTerm(Phrase,Score)) ;

		}
	}

	ErrorCode = ExtrClearDocumentMemory(DocumentMemory);
	if (ErrorCode > 0) {
		ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
	       	//AddToLog (5, ErrorMessage);
		return false ;
	}

	ErrorCode = ExtrClearStopMemory(StopMemory);
	if (ErrorCode > 0) {
		ExtrGetErrorMessage(ErrorCode, &ErrorMessage);
	       	//AddToLog (5, ErrorMessage);
		return false ;
	}

	return allWentWell ;
}
char* HtmlPage::IncreaseArray(char *TempStream, int Chunk)
{
	int i=0;
	char *tempA;
	tempA=new char[Chunk*1000];
	int StrLeng=strlen( TempStream );
	for ( i=0 ; i < StrLeng ; i++)
		tempA[i]=TempStream[i];
	delete []TempStream;
	return tempA;
}

char* HtmlPage::ReadFile(string FilePath)
{
	FILE *fp;
	char *TempStream;
	char w;
	int i=0;
	int ArraySize=1;
	TempStream=new char[1000];
	fp=fopen( FilePath.c_str() , "r" );
	while( (fscanf(fp,"%c",&w)) != EOF )
	{
		TempStream[i]=w;	
		if ( i == ArraySize*1000-2 )
		{
			TempStream[i+1]='\0';
			TempStream=IncreaseArray( TempStream , ++ArraySize);
		}
		i++;			
	}
	fclose(fp);
	TempStream[i]='\0';
	return TempStream;
}


void HtmlPage::AddPage(string PageUrl)
{
	char *stream = ReadFile( PageUrl.c_str() );
	GenerateTermsList(stream);
	delete stream;
}
typedef 	vector<string> VecF;
int _tmain(int argc, TCHAR* argv[], TCHAR* envp[])
{
	int nRetCode = 0;
	char *p;
	int DirOpen;
	VecF vecFiles;
	NumOfPhrases=1;
	HtmlPage *pHtmlPage=NULL;
	char DirPath[300];
	DirPath[0]='\0';
	// initialize MFC and print and error on failure
	if (!AfxWinInit(::GetModuleHandle(NULL), NULL, ::GetCommandLine(), 0))
	{
		// TODO: change error code to suit your needs
		cerr << _T("Fatal Error: MFC initialization failed") << endl;
		nRetCode = 1;
	}
	else
	{
		// TODO: code your application's behavior here.
		CString strHello;
		strHello.LoadString(IDS_HELLO);
		cout << (LPCTSTR)strHello << endl;
	}
//	p=ReadFile("e:\\CNN_com.htm");
//	GenerateTermsList(p);
	cout<<"please enter the web page's directory path\n";
	cin>>DirPath;
	int iRC = SearchDirectory(vecFiles, DirPath , "html");
	iRC = SearchDirectory(vecFiles, DirPath, "htm");
	VecF::iterator posFilesNames;
	for ( posFilesNames = vecFiles.begin() ; posFilesNames != vecFiles.end() ; posFilesNames++)
	{
		pHtmlPage=new HtmlPage;
		cout<<posFilesNames->c_str()<<endl;
		pHtmlPage->AddPage( *posFilesNames );
		AllPagesVector.push_back( *pHtmlPage );
	}
	return nRetCode;
}


